export PYTHON_ROOT=~/Python
export PYSPARK_DRIVER_PYTHON=/home/hdfs/Python/bin/python3
export PYSPARK_PYTHON=/home/hdfs/Python/bin/python3
export LD_LIBRARY_PATH=${PATH}
export PYSPARK_PYTHON=${PYTHON_ROOT}/bin/python3
export SPARK_YARN_USER_ENV="PYSPARK_PYTHON=/home/hdfs/Python/bin/python3"
export PATH=${PYTHON_ROOT}/bin/:$PATH
export QUEUE=default
export LIB_HDFS=$HADOOP_HOME/lib/native             
export LIB_JVM=$JAVA_HOME/jre/lib/amd64/server
export HADOOP_PREFIX=$HADOOP_HOME
${SPARK_HOME}/bin/spark-submit \
--master yarn \
--deploy-mode cluster \
--queue ${QUEUE} \
--num-executors 2 \
--executor-memory 4G \
--py-files hdfs:///user/${USER}/tfspark.zip,${TFoS_HOME}/examples/mnist/spark/mnist_dist.py \
--conf spark.dynamicAllocation.enabled=false \
--conf spark.yarn.maxAppAttempts=1 \
--conf spark.executorEnv.LD_LIBRARY_PATH=$LIB_JVM:$LIB_HDFS \
--driver-library-path=$LIB_JVM:$LIB_HDFS \
--archives hdfs:///user/${USER}/Python.zip#Python \
${TFoS_HOME}/examples/mnist/spark/mnist_spark.py \
--images mnist/csv/train/images \
--labels mnist/csv/train/labels \
--mode train \
--model mnist_model

